import scrapy
from scrapy.selector import Selector
from douban_movie_top250.items import DoubanMovieTop250Item
import re

class DoubanSpider(scrapy.Spider):
    name = 'douban_movie'
    allowed_domains = ['movie.douban.com']
    start_urls = ['https://movie.douban.com/top250']
    
    def parse(self, response):
        """解析电影列表页"""
        self.logger.info(f"正在解析页面: {response.url}")
        
        # 获取电影列表
        movie_items = response.xpath('//ol[@class="grid_view"]/li')
        
        for item in movie_items:
            # 创建电影数据项
            movie = DoubanMovieTop250Item()
            
            # 提取基本信息
            movie['ranking'] = int(item.xpath('.//em/text()').get())
            movie['title'] = item.xpath('.//span[@class="title"][1]/text()').get()
            
            # 提取其他标题
            other_titles = item.xpath('.//span[@class="other"]/text()').get()
            if other_titles:
                movie['title'] += other_titles.replace('/\xa0', ' ')
            
            # 提取评分和评价人数
            movie['rating'] = float(item.xpath('.//span[@class="rating_num"]/text()').get())
            rating_num_text = item.xpath('.//div[@class="star"]/span[last()]/text()').get()
            movie['rating_num'] = int(re.search(r'(\d+)', rating_num_text).group(1))
            
            # 提取电影链接和ID
            movie_link = item.xpath('.//div[@class="pic"]/a/@href').get()
            movie['douban_id'] = re.search(r'subject/(\d+)', movie_link).group(1)
            
            # 提取电影海报
            movie['poster_url'] = item.xpath('.//div[@class="pic"]/a/img/@src').get()
            
            # 提取电影描述
            description = item.xpath('.//span[@class="inq"]/text()').get()
            movie['description'] = description if description else ''
            
            # 请求电影详情页
            yield scrapy.Request(
                url=movie_link,
                callback=self.parse_movie_detail,
                meta={'movie': movie}
            )
            
        # 提取下一页链接
        next_page = response.xpath('//span[@class="next"]/a/@href').get()
        if next_page:
            next_page_url = response.urljoin(next_page)
            self.logger.info(f"准备爬取下一页: {next_page_url}")
            yield scrapy.Request(url=next_page_url, callback=self.parse)
            
    def parse_movie_detail(self, response):
        """解析电影详情页"""
        self.logger.info(f"正在解析电影详情页: {response.url}")
        
        # 获取从列表页传递过来的电影数据
        movie = response.meta['movie']
        
        # 提取详细信息
        info = response.xpath('//div[@id="info"]').get()
        info_selector = Selector(text=info)
        
        # 提取导演信息
        directors = info_selector.xpath('//span[text()="导演"]/following-sibling::span[1]//a/text()').getall()
        movie['directors'] = '/'.join(directors) if directors else ''
        
        # 提取演员信息
        actors = info_selector.xpath('//span[text()="主演"]/following-sibling::span[1]//a/text()').getall()
        movie['actors'] = '/'.join(actors) if actors else ''
        
        # 提取类型
        genres = info_selector.xpath('//span[text()="类型:"]/following-sibling::text()').get()
        if genres:
            movie['genres'] = genres.strip()
        
        # 提取制片国家/地区
        country = info_selector.xpath('//span[text()="制片国家/地区:"]/following-sibling::text()').get()
        if country:
            movie['country'] = country.strip()
        
        # 提取语言
        language = info_selector.xpath('//span[text()="语言:"]/following-sibling::text()').get()
        if language:
            movie['language'] = language.strip()
        
        # 提取上映日期
        release_date = info_selector.xpath('//span[text()="上映日期:"]/following-sibling::text()').get()
        if release_date:
            movie['release_date'] = release_date.strip()
        
        # 提取片长
        runtime_text = info_selector.xpath('//span[text()="片长:"]/following-sibling::text()').get()
        if runtime_text:
            runtime_match = re.search(r'(\d+)', runtime_text)
            if runtime_match:
                movie['runtime'] = int(runtime_match.group(1))
        
        # 提取年份
        year_text = response.xpath('//h1/span[@class="year"]/text()').get()
        if year_text:
            year_match = re.search(r'(\d+)', year_text)
            if year_match:
                movie['year'] = int(year_match.group(1))
        
        # 返回完整的电影数据
        yield movie